{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# forward model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import h5py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "params = {\n",
    "    'model_weight' : '../ner_model_weight/model_conv_625.h5',\n",
    "    'embed_size' : 500,\n",
    "    'max_sent_len': 20\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "with open('../char_6.17.json', mode='r', encoding='utf-8') as f:\n",
    "    dicts = json.load(f)\n",
    "char2id = dicts['char2id']\n",
    "id2char = dicts['id2char']\n",
    "intent2id = dicts['intent2id']\n",
    "id2intent = dicts['id2intent']\n",
    "slot2id = dicts['slot2id']\n",
    "id2slot = dicts['id2slot']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sigmoid(x):\n",
    "    y = 1 / (1 + np.exp(-x))\n",
    "    return y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def tanh(x):\n",
    "    y=(np.exp(x)-np.exp(-x))/(np.exp(x)+np.exp(-x))\n",
    "    return y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def embedding(x,embed_size,embed):\n",
    "    x_one= np.zeros((len(x),embed_size))\n",
    "    x_one[range(len(x)), x] = 1\n",
    "    x_embed = np.dot(x_one, embed)\n",
    "    return x_embed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def GRU(cell_inputs,cell_states,kernel,recurrent_kernel,bias):\n",
    "    input_bias = bias[0]\n",
    "    recurrent_bias = bias[1]\n",
    "    \n",
    "    matrix_x = np.dot(cell_inputs, kernel)\n",
    "    matrix_x = np.add(matrix_x, input_bias)\n",
    "    \n",
    "    x_z, x_r, x_h = np.split(matrix_x,3, axis=-1)\n",
    "    \n",
    "    matrix_inner = np.dot(cell_states, recurrent_kernel)\n",
    "    matrix_inner = np.add(matrix_inner, recurrent_bias)\n",
    "    \n",
    "    recurrent_z, recurrent_r, recurrent_h = np.split(matrix_inner,3,axis=-1)\n",
    "    \n",
    "    z = sigmoid(x_z + recurrent_z)\n",
    "    r = sigmoid(x_r + recurrent_r)\n",
    "    hh = tanh(x_h + r * recurrent_h)\n",
    "    \n",
    "    h = z * cell_states + (1-z) * hh\n",
    "    return h"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def GlobalAveragePooling1D(x,step_axis=0):\n",
    "    return np.mean(x,axis=step_axis)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def LayerNormalization(x,gamma,beta,step_axis = -1,epsilon=1e-3):\n",
    "    mean = np.mean(x,axis = step_axis)\n",
    "    mean = np.expand_dims(mean,axis=1)\n",
    "    variance = np.var(x,axis = step_axis)\n",
    "    variance = np.expand_dims(variance,axis=1)\n",
    "    inv = 1.0 / np.sqrt(variance + epsilon)\n",
    "#     print(np.shape(inv))\n",
    "    gamma = np.expand_dims(gamma,axis=0)\n",
    "    beta = np.expand_dims(beta,axis=0)\n",
    "    inv = gamma *inv\n",
    "    return x * inv + (beta - mean * inv)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def dense(x,gamma, bias):\n",
    "#     print(np.shape(x))\n",
    "    y = np.matmul(x,gamma)\n",
    "    y = np.add(y,bias)\n",
    "    return y "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_weight(file_path):\n",
    "    f = h5py.File(file_path,'r')\n",
    "    \n",
    "    embed = f['embedding']['embedding']['embeddings:0'][:]\n",
    "    \n",
    "    forword_gru_bias = f['bidirectional']['bidirectional']['forward_gru']['gru_cell_1']['bias:0'][:]\n",
    "    forword_gru_kernel = f['bidirectional']['bidirectional']['forward_gru']['gru_cell_1']['kernel:0'][:]\n",
    "    forword_gru_recurrent_kernel = f['bidirectional']['bidirectional']['forward_gru']['gru_cell_1']['recurrent_kernel:0'][:]\n",
    "    \n",
    "    backward_gru_bias = f['bidirectional']['bidirectional']['backward_gru']['gru_cell_2']['bias:0'][:]\n",
    "    backward_gru_kernel = f['bidirectional']['bidirectional']['backward_gru']['gru_cell_2']['kernel:0'][:]\n",
    "    backward_gru_recurrent_kernel = f['bidirectional']['bidirectional']['backward_gru']['gru_cell_2']['recurrent_kernel:0'][:]\n",
    "    \n",
    "    layer_normal_beta = f['layer_normalization']['layer_normalization']['beta:0'][:]\n",
    "    layer_normal_gamma = f['layer_normalization']['layer_normalization']['gamma:0'][:]\n",
    "    \n",
    "    layer_normal_beta1 = f['layer_normalization_1']['layer_normalization_1']['beta:0'][:]\n",
    "    layer_normal_gamma1 = f['layer_normalization_1']['layer_normalization_1']['gamma:0'][:]\n",
    "    \n",
    "    pre_intent_bias = f['pre_intent']['pre_intent']['bias:0'][:]\n",
    "    pre_intent_gamma = f['pre_intent']['pre_intent']['kernel:0'][:]\n",
    "    \n",
    "    pre_ner_bias = f['pre_ner']['pre_ner']['bias:0'][:]\n",
    "    pre_ner_gamma = f['pre_ner']['pre_ner']['kernel:0'][:]\n",
    "    \n",
    "    return embed, forword_gru_bias,forword_gru_kernel,forword_gru_recurrent_kernel, backward_gru_bias,backward_gru_kernel,backward_gru_recurrent_kernel,\\\n",
    "            layer_normal_beta,layer_normal_gamma,layer_normal_beta1,layer_normal_gamma1,pre_intent_bias,pre_intent_gamma,pre_ner_bias,pre_ner_gamma"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def trans2labelid(vocab, labels, max_sent_len):\n",
    "    labels = [vocab[label] for label in labels]\n",
    "    if len(labels) < max_sent_len:\n",
    "        labels += [0] * (max_sent_len - len(labels))\n",
    "    else:\n",
    "        labels = labels[:max_sent_len]\n",
    "    return labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "def test(inputs,h5file_path):\n",
    "    embed, forword_gru_bias,forword_gru_kernel,forword_gru_recurrent_kernel, backward_gru_bias,backward_gru_kernel,backward_gru_recurrent_kernel,\\\n",
    "            layer_normal_beta,layer_normal_gammma,layer_normal_beta1,layer_normal_gamma1,pre_intent_bias,pre_intent_gamma,pre_ner_bias,pre_ner_gamma = get_weight(h5file_path)\n",
    "    forword_gru_state = np.zeros((64))\n",
    "    backed_gru_state = np.zeros((64))\n",
    "    x = trans2labelid(char2id,inputs,params['max_sent_len'])\n",
    "    print(x)\n",
    "    embed = embedding(x,params['embed_size'], embed)\n",
    "#     print('embed: {}'.format(embed))\n",
    "    recurrent_embed = embed[::-1]\n",
    "    forword_gru = []\n",
    "    backed_gru = []\n",
    "#     print(np.shape(recurrent_embed))\n",
    "    for i in range(len(x)):\n",
    "#         print(np.shape(embed[i]))\n",
    "        forward_h = GRU(embed[i],forword_gru_state,forword_gru_kernel,forword_gru_recurrent_kernel,forword_gru_bias)\n",
    "        backed_h =  GRU(recurrent_embed[i], backed_gru_state,backward_gru_kernel,backward_gru_recurrent_kernel,backward_gru_bias)\n",
    "        forword_gru_state = forward_h\n",
    "        backed_gru_state = backed_h\n",
    "        forword_gru.append(forward_h)\n",
    "        backed_gru.append(backed_h)\n",
    "    gru_out = np.concatenate((forword_gru,backed_gru[::-1]), axis=-1)\n",
    "#     print('gru_out: {}'.format(gru_out))\n",
    "#     print(np.shape(gru_out))\n",
    "    x_in = LayerNormalization(gru_out,layer_normal_gammma,layer_normal_beta)\n",
    "#     print(np.shape(x_in))\n",
    "#     print('x_in : ',x_in)\n",
    "    x_conv = GlobalAveragePooling1D(x_in)\n",
    "#     print(np.shape(x_conv))\n",
    "    pre_intent = dense(x_conv,pre_intent_gamma,pre_intent_bias)\n",
    "    pre_intent = sigmoid(pre_intent)\n",
    "    \n",
    "    x_ner = LayerNormalization(gru_out,layer_normal_gamma1,layer_normal_beta1)\n",
    "    pre_slot = dense(x_ner,pre_ner_gamma,pre_ner_bias)\n",
    "    pre_slot = sigmoid(pre_slot)\n",
    "    \n",
    "    return embed,gru_out,x_in,x_conv,x_ner,pre_intent,pre_slot\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "inputs = '打开空调'\n",
    "h5file_path = '../ner_model_weight/model_conv_625.h5'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "embed_np,gru_out_np,x_in_np,x_conv_np,x_ner_np,pre_intent_np,pre_slot_np = test(inputs,h5file_path) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 前向与原始模型对比验证"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "params['intent_num'] = len(intent2id)\n",
    "params['slot_num'] = len(slot2id)\n",
    "params['id2intent'] = id2intent\n",
    "params['id2slot'] = id2slot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"functional_1\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "Input (InputLayer)              [(None, 20)]         0                                            \n",
      "__________________________________________________________________________________________________\n",
      "embedding (Embedding)           (None, 20, 32)       16000       Input[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "bidirectional (Bidirectional)   (None, 20, 128)      37632       embedding[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "layer_normalization (LayerNorma (None, 20, 128)      256         bidirectional[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "global_average_pooling1d (Globa (None, 128)          0           layer_normalization[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "layer_normalization_1 (LayerNor (None, 20, 128)      256         bidirectional[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "pre_intent (Dense)              (None, 55)           7095        global_average_pooling1d[0][0]   \n",
      "__________________________________________________________________________________________________\n",
      "pre_ner (Dense)                 (None, 20, 36)       4644        layer_normalization_1[0][0]      \n",
      "==================================================================================================\n",
      "Total params: 65,883\n",
      "Trainable params: 65,883\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "tf.keras.backend.clear_session()\n",
    "text_inputs = tf.keras.layers.Input(shape=(20,),name='Input')\n",
    "embed = tf.keras.layers.Embedding(500,32)(text_inputs)\n",
    "bilstm = tf.keras.layers.Bidirectional(tf.keras.layers.GRU(64,return_sequences=True))(embed)\n",
    "x_in = tf.keras.layers.LayerNormalization()(bilstm)\n",
    "x_conv = tf.keras.layers.GlobalAveragePooling1D()(x_in)\n",
    "pre_intent = tf.keras.layers.Dense(params['intent_num'],\\\n",
    "            activation='sigmoid',name = 'pre_intent')(x_conv)\n",
    "x_ner  = tf.keras.layers.LayerNormalization()(bilstm)\n",
    "pre_slot = tf.keras.layers.Dense(params['slot_num'],activation='sigmoid',name = 'pre_ner')(x_ner)\n",
    "model = tf.keras.Model(text_inputs,[embed,bilstm,x_in,x_conv,x_ner,pre_intent,pre_slot])\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.load_weights('../ner_model_weight/model_conv_625.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "x = trans2labelid(char2id,inputs,params['max_sent_len'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "embed,bilstm,x_in,x_conv,x_ner,pre_intent,pre_slot = model.predict([x])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
